//************************************************************
//    > File Name: lanvshen.cpp
//    > Author: jiayue
//    > Mail: tjyemail@sina.com 
//    > Created Time: Sat Mar 07 09:40:56 2020
//***********************************************************

#include<bits/stdc++.h>
#include<unistd.h>
#include<dirent.h>

#define LANVSHEN_NAME		"Archive Web-Pic from www.lanvshen.com"
#define LANVSHEN_VERSION	"0.4"
#define LANVSHEN_DATE		"2020.04.07"


#ifdef __MINGW32__
#include<io.h>
#include<windows.h>
#define LANVSHEN_OS "MINGW32"
static const char CMD_MV[] = "mv";
static const char CMD_RM[] = "rm";

#elif __linux__
#define LANVSHEN_OS "Linux"
static const char CMD_MV[] = "mv";
static const char CMD_RM[] = "rm";

#elif __WIN32

#endif

using namespace std;

#define LNS_PATH_LENGTH 4096
static const char LNS_PATH_ILLEGAL[] = "/\\:,;?*\"\'<>|& \n\r";

//---------------- utility function -----------------

static void trim_end(char *str, const char trim)
{
	for(int i=strlen(str); i-- && str[i]==trim; str[i]='\0');
}

static void trim_end(string &str, const char trim)
{
	for(; str.length() && str.back()==trim; str.pop_back());
}

static void trim_begin(char *str, const char trim)
{
	char *p = str;
	for(; p && *p==trim; ++p);
	memmove(str, p, strlen(p)+1);
}

static void trim_begin(string &str, const char trim)
{
	int offset = 0;
	for(; offset<str.length() && str[offset]==trim; ++offset);
	str = offset>=str.length() ? "" : str.substr(offset);
}

static void replace_char(char *str, const char org, const char dest)
{
	for(; str && *str!='\0'; ++str)
		if (*str==org) *str=dest;
}

static void remove_str(char *name, const char *illegal)
{
	char *p = name;
	for(; p && *p!='\0'; ++p){
		if (!strchr(illegal, *p)){
			*name = *p;
			++name;
		}
	}
	if(name) *name = '\0';
}

static void remove_str(string &name, const char *illegal)
{
	char tmp[LNS_PATH_LENGTH];
	memset(tmp, 0, sizeof(tmp));
	strncpy(tmp, name.c_str(), sizeof(tmp)-1);
	remove_str(tmp, illegal);
	name = tmp;
}

static void remove_str(string &name, const string &illegal)
{
	remove_str(name, illegal.c_str());
}

#ifdef _WIN32
static void utf8_to_gbk(const char *org, char *dest, int size)
{
	wchar_t buf[LNS_PATH_LENGTH*4];
	memset(buf, 0, sizeof(buf));
	MultiByteToWideChar(CP_UTF8, 0, org, -1, buf, sizeof(buf)/sizeof(wchar_t)-1);
	memset(dest, 0, size);
	WideCharToMultiByte(CP_ACP, 0, buf, -1, dest, size, NULL, NULL);
}

static void utf8_to_gbk(char *str, int size=0)
{
	wchar_t buf[LNS_PATH_LENGTH*4];
	memset(buf, 0, sizeof(buf));
	if(size==0) size = strlen(str);
	MultiByteToWideChar(CP_UTF8, 0, str, -1, buf, sizeof(buf)/sizeof(wchar_t)-1);
	memset(str, 0, size);
	WideCharToMultiByte(CP_ACP, 0, buf, -1, str, size, NULL, NULL);
}
#endif

static int file_exists(const char *file)
{
	if (access(file, R_OK)==0) return 1;
	else return 0;
	/*
	FILE *fp = fopen(file, "r");
	if (fp==NULL) return 0;
	fclose(fp);
	return 1;
	*/
}

static long file_bytes(const char *file)
{
	FILE *fp = fopen(file, "r");
	long bytes = 0;
	if (fp==NULL) return 0;
	fseek(fp, 0L, SEEK_END);
	bytes = ftell(fp);
	fclose(fp);
	return bytes;
}

static int systemcmd(const char *cmd, const char os='w')
{
#ifdef _WIN32
	char buf[LNS_PATH_LENGTH*4];
	memset(buf, 0, sizeof(buf));
	strncpy(buf, cmd, sizeof(buf)-1);
	if (os=='w') replace_char(buf, '/', '\\');
	utf8_to_gbk(buf, buf, sizeof(buf));
	system(buf);
#else
	system(cmd);
#endif
	return 0;
}


// ------------------- global params --------------------

static char lns_list_file[LNS_PATH_LENGTH]		= "";
static char lns_list_dir[LNS_PATH_LENGTH]		= "";
static char lns_list_exclude[LNS_PATH_LENGTH]	= "";
static char lns_list_type = 'p';	// default download project[page]
static char lns_do_wget = 't';		// default do download
static unordered_map<string, unordered_set<string> > lns_exclude_dict;	// prefix, url

static void show_version()
{
	printf("%s\nversion %s, %s on %s\n", LANVSHEN_NAME,
			LANVSHEN_VERSION, LANVSHEN_DATE, LANVSHEN_OS);
}

static void show_help(const char *tag=NULL)
{
	if (tag){
		printf("Unknown command: %s , -h for help\n", tag);
		return;
	}

	printf("Usages:\n");
	printf(" -i  url.txt    echo line is url\n");
	printf("[-o] dir        tsave to dir\n");
	printf("[-e] url.txt    exclude url, not download\n");
	printf("[-t] [m/p/t]    url type: [m]odel, [p]rpject, [t]ype\n");
	printf("                default is [p]roject\n");
	printf("[-d] [t/f]      do wget: [t]rue, [f]alse. default [t]rue\n");
	printf("[-h]            show this help\n");

	printf("Examples:\n");
	printf("lns -i url.txt -t m\n");
	printf("lns -i https://www.abc.com/t/a/123 -t p\n");

}

static int parser_param(int argc, char *argv[])
{
	static const char params[] = "ioethd";
	static const char types[] = "mpt";
	for(int i=1; i<argc; ++i){
		if (argv[i][0]!='-' || strlen(argv[i])!=2 || !strchr(params, argv[i][1])){
			printf("unknown command %s, -h for help\n", argv[i]);
			return i;
		}
		if (argv[i][1]=='h'){
			show_help();
			return i;
		}
		if (i+1==argc){
			printf("missing param %s, -h for help\n", argv[i]);
			return i;
		}
		switch(argv[i++][1]){
			case 'i':
				sprintf(lns_list_file, "%s", argv[i]);
				break;
			case 'o':
				sprintf(lns_list_dir, "%s", argv[i]);
				break;
			case 'e':
				sprintf(lns_list_exclude, "%s", argv[i]);
				break;
			case 't':
				if (!strchr(types, argv[i][0])){
					printf("unknown param %s, -h for help\n", argv[i]);
					return i;
				}
				lns_list_type = argv[i][0];
				break;
			case 'd':
				lns_do_wget = argv[i][0];
				break;
		}
	}

	trim_end(lns_list_file, '\\');
	trim_end(lns_list_file, '/');
	if (strlen(lns_list_file)==0){
		printf("please input url file, -h for help\n");
		return -1;
	}
	if (strlen(lns_list_dir)==0){
		strcpy(lns_list_dir, lns_list_file);
		char *p = strrchr(lns_list_dir, '.');
		if (p) *p='\0';
	}
	trim_end(lns_list_dir, '\\');
	trim_end(lns_list_dir, '/');
	return 0;
}


// -------------- declare function -------------------

static int lns_ifs_getline(ifstream &ifs, string &line);
static vector<string> lns_read_file(const char *file);
static vector<string> lns_load_urls(const char *file);
static void lns_load_exclude(const char *file);
static int lns_exclude_contains(const string &line);
static void lns_record_project(const char *info, int append=1);
static int download_http_file(const char *url, const char *file);
static int lns_download_list(const vector<string> &urls, const char *list_dir);
static int lns_parser_project(const char *html_file, const char *html_dir, int photo=0);
static int lns_parser_model(const char *html_file, const char *html_dir);


// -------------- define function ----------------------

static int lns_ifs_getline(ifstream &ifs, string &line)
{
	for(char buf[10240]; !ifs.eof(); ){
		ifs.getline(buf, sizeof(buf));
		trim_end(buf, ' ');
		trim_begin(buf, ' ');
		line = buf;
		if (line.length()>0) return 1;
	}
	return 0;
}

static vector<string> lns_read_file(const char *file)
{
	cout<<"========= read [FILE] "<<file<<" =========\n";
	vector<string> vs;
	if (file_exists(file)==0){
		cout<<"====== file "<<file<<" can't read ! ==== "<<endl;
		return vs;
	}

	ifstream ifs(file);
	for(string line; lns_ifs_getline(ifs, line);){
		trim_begin(line, '\r');
		trim_end(line, '\r');
		if (line.length()==0) continue;
		vs.push_back(line);
	}
	ifs.close();
	return vs;
}

static vector<string> lns_load_urls(const char *file)
{
	cout<<"========= load url list ["<<file<<"] =========\n";
	vector<string> vs;
	if (file==NULL) return vs;
	if (strncmp(file, "http://", 7)==0 || strncmp(file, "https://", 8)==0){
		vs.push_back(file);
		return vs;
	}
	vector<string> lines = lns_read_file(file);
	for(string &line:lines){
		if (line[0]=='#') continue;
		trim_end(line, '/');
		// TODO : check URL
		vs.push_back(line);
	}
	return vs;
}

static void lns_load_exclude(const char *file)
{
	cout<<"========= load exclude list [FILE] "<<file<<" =========\n";
	vector<string> vs = lns_load_urls(file);
	unordered_map<string, unordered_set<string> > &dict = lns_exclude_dict;
	for(string &line:vs){
		string prefix = line.substr(0, line.rfind('/'));
		string suffix = line.substr(line.rfind('/')+1);
		dict[prefix].insert(suffix);
	}
}

static int lns_exclude_contains(const string &line)
{
	unordered_map<string, unordered_set<string> > &dict = lns_exclude_dict;
	string prefix = line.substr(0, line.rfind('/'));
	string suffix = line.substr(line.rfind('/')+1);
	int count = dict[prefix].count(suffix);
	dict[prefix].insert(suffix);	// add current download item
	return count;
}

static int download_http_file(const char *url, const char *file)
{
	cout<<"========== download [URL] "<<url<<" ---> [FILE] "<<file<<endl;
	char cmd[LNS_PATH_LENGTH*4];
	char wget[] = "wget --no-check-certificate -q ";
	sprintf(cmd, "%s %s -O %s",
			wget, url, file);
	printf("CMD: %s\n", cmd);
#ifdef _WIN32
	utf8_to_gbk(cmd, cmd, sizeof(cmd));
#endif
	//if (file_exists(file)) return 1;
	//else return 0;
	long MIN_BYTES = 1024;
	long bytes = 0;
	int RETRY = 5;
	for(int retry=0; bytes<MIN_BYTES && retry<RETRY; ++retry){
		if (retry){
			//system("sleep 2");		// linux
			//system("timeout /T 2");	// windows
			sleep(2);
		}
		system(cmd);
		bytes = file_bytes(file);
	}
	return bytes;
}

static int lns_download_list(const vector<string> &urls, const char *list_dir)
{
	cout<<"========= download url list to [DIR] "<<list_dir<<" =========\n";
	if (urls.size()==0) return 0;
	char cmd[LNS_PATH_LENGTH*4];
	int makeddir = 0;
	if (access(list_dir, W_OK)!=0){
		sprintf(cmd, "mkdir %s", list_dir);
		printf("==== [CMD] %s\n", cmd);
		systemcmd(cmd);
		makeddir = 1;
	}

	int count = 0;
	int subcount = 0 ;
	int id = 0;
	for(const string &url:urls){
		if (lns_exclude_contains(url)) {
			++id;
			continue;
		}

		string dir = url;
		remove_str(dir, LNS_PATH_ILLEGAL);
		dir = "/" + to_string(id) + "_" + dir;
		dir.insert(0, list_dir);
		switch(lns_list_type){
			case 'm':
				subcount += lns_parser_model(url.c_str(), dir.c_str());
				break;
			case 'p':
				subcount += lns_parser_project(url.c_str(), dir.c_str(), 0);
				break;
			case 't':
				// TODO : tag, series
				break;
		}
		++count;
		++id;
	}

	cout<<"========= total download "<<lns_list_type<<" "<<count<<endl;
	if (subcount==0 && makeddir){
		// has maked dir, but download null, rm dir
		sprintf(cmd, "rm -rf %s", list_dir);
		systemcmd(cmd, 'l');
	}
	return count;
}

static void lns_record_project(const char *info, int append)
{
	string logfile = lns_list_file;
	remove_str(logfile, LNS_PATH_ILLEGAL);
	logfile = logfile + ".log";
	//cout<<"============ record [LOG] "<<info<<" ===========\n";
	ofstream ofs(logfile.c_str(), append ? ios::app : ios::out);
	if (ofs){
		ofs<<info<<endl;
		ofs.close();
	}
}


// parser model html and download each project[page]
static int lns_parser_model(const char *html_url, const char *html_dir)
{
	cout<<"========= parser model html ["<<html_url<<"] =========\n";
	char cmd[LNS_PATH_LENGTH*4];
	int makeddir = 0;
	if (access(html_dir, W_OK)!=0){
		sprintf(cmd, "mkdir %s", html_dir);
		printf("==== [CMD] %s\n", cmd);
		systemcmd(cmd);
		makeddir = 1;
	}
	
	string html_file = html_dir;
	html_file += "/index.html";
	download_http_file(html_url, html_file.c_str());
	
	string model_dir;
	int project_count = 0;
	int photo_count = 0;
	int count = 0;

	vector<string> urls, dirs;
	vector<int> counts;
	for(const string &line:lns_read_file(html_file.c_str())){
		if (line.length()==0) continue;
		// find model name
		if (line.find("<title>")!=string::npos &&
			line.find("</title>")!=string::npos){
			cout<<"---- READ ["<<html_file<<"] LINE ["<<line<<"]\n";
			string title = line.substr(line.find("<title>")+7);
			title = title.substr(0, title.find("</title>"));
			remove_str(title, LNS_PATH_ILLEGAL);
			model_dir = html_dir;
			model_dir = model_dir.substr(0, model_dir.rfind('/'));
			model_dir = model_dir+"/"+title;
			continue;
		}
		// find project count
		if (line.find("shoulushuliang")!=string::npos &&
			line.find("<span>")!=string::npos &&
			line.find("</span>")!=string::npos){
			cout<<"---- READ ["<<html_file<<"] LINE ["<<line<<"]\n";
			string pcline = line.substr(line.find("<span>")+6);
			pcline = pcline.substr(0, pcline.find("</span>"));
			sscanf(pcline.c_str(), "%d", &project_count);
			cout<<"---- count "<<pcline<<", atoi = "<<project_count<<endl;
			continue;
		}
		// find project
		if (line.find("=\"biaoti\">")!=string::npos &&
			line.find("http")!=string::npos &&
			line.find("target=")!=string::npos){
			cout<<"---- READ ["<<html_file<<"] LINE ["<<line<<"]\n";
			string url = line.substr(line.find("<a href=")+8);
			url = url.substr(url.find("http"));
			if (url.find("\"")!=string::npos)
				url = url.substr(0, url.find("\""));
			trim_end(url, '/');
			// downlaod project
			string dir = url;
			remove_str(dir, LNS_PATH_ILLEGAL);
			dir = "/" + to_string(count) + "_" + dir;
			dir.insert(0, html_dir);
			urls.push_back(url);
			dirs.push_back(dir);
			counts.push_back(photo_count);
			++count;
		}
		// find photo count
		if (line.find("shuliang")!=string::npos){
			cout<<"---- READ ["<<html_file<<"] LINE ["<<line<<"]\n";
			string phcline = line.substr(line.find("shuliang\">")+10);
			if (phcline.find("</span>")!=string::npos)
				phcline = phcline.substr(0, phcline.find("</span>"));
			trim_end(phcline, 'P');
			trim_end(phcline, 'p');
			sscanf(phcline.c_str(), "%d", &photo_count);
			cout<<"---- count "<<phcline<<", atoi = "<<photo_count<<endl;
		}
	}
	count = 0;
	int subcount = 0;
	for(int i=0; i<urls.size(); ++i){
		string url = urls[i], dir = dirs[i];
		if (lns_exclude_contains(url)) continue;
		//sprintf(cmd, "mkdir %s", dir.c_str());
		//printf("==== [CMD] %s\n", cmd);
		//systemcmd(cmd);
		//string file = dir+"/index.html";
		int photo = counts[i];
		//download_http_file(url.c_str(), file.c_str());
		subcount += lns_parser_project(url.c_str(), dir.c_str(), photo);
		++count;
	}
	cout<<"===== total download project ["<<count<<"/"<<project_count<<"]\n";
	if (subcount==0 && makeddir){
		// has maked dir, but download null, rm dir
		sprintf(cmd, "rm -rf %s", html_dir);
		systemcmd(cmd, 'l');
	}
	else {
		// rename model dir
		sprintf(cmd, "mv %s %s", html_dir, model_dir.c_str());
		printf("===== [CMD] %s\n", cmd);
		systemcmd(cmd, 'l');
	}
	return count;
}

// parser project html and download each photo
static int lns_parser_project(const char *html_url, const char *html_dir, int photo)
{
	cout<<"========= parser project html ["<<html_url<<"] =========\n";
	char cmd[LNS_PATH_LENGTH*4];
	int makeddir = 0;
	if (access(html_dir, W_OK)!=0){
		sprintf(cmd, "mkdir %s", html_dir);
		printf("==== [CMD] %s\n", cmd);
		systemcmd(cmd);
		makeddir = 1;
	}
	
	string html_file = html_dir;
	html_file += "/index.html";
	download_http_file(html_url, html_file.c_str());
	
	string project_dir;
	int count = 0;
	string photo_url;
	for(const string &line:lns_read_file(html_file.c_str())){
		if (line.length()==0) continue;
		// find title
		if (project_dir.length()==0 &&
			line.find("<title>")!=string::npos &&
			line.find("</title>")!=string::npos){
			cout<<"---- READ ["<<html_file<<"] LINE ["<<line<<"]\n";
			string title = line.substr(line.find("<title>")+7);
			title = title.substr(0, title.find("</title>"));
			remove_str(title, LNS_PATH_ILLEGAL);
			project_dir = html_dir;
			project_dir = project_dir.substr(0, project_dir.rfind('/'));
			project_dir = project_dir+"/"+title;
			// get photo count. [16P]_
			if (photo==0 && title.find('[')!=string::npos){
				string ph_c = title.substr(title.rfind('[')+1);
				ph_c = ph_c.substr(0, ph_c.find(']'));
				trim_end(ph_c, 'P');
				trim_end(ph_c, 'p');
				int c = 0;
				sscanf(ph_c.c_str(), "%d", &c);
				cout<<"---- photo count "<<ph_c<<", atoi = "<<c<<endl;
				if (c>0 && c<9999) photo = c;
			}
			continue;
		}
		// find photo url
		if (line.find("<img src=")==0 &&
			line.find("http")!=string::npos){
			cout<<"---- READ ["<<html_file<<"] LINE ["<<line<<"]\n";
			// found some url, and calculate photo url. TODO : locate each page and get photo url.
			photo_url = line.substr(line.find("http"));
			if (photo_url.find("\"")!=string::npos)
				photo_url = photo_url.substr(0, photo_url.find("\""));
			cout<<"---- get photo url "<<photo_url<<endl;
			//break;
		}
	}

	if (photo_url.length()<1) {
		if (makeddir){
			// has maked dir, but download null, rm dir
			sprintf(cmd, "rm -rf %s", html_dir);
			systemcmd(cmd, 'l');
		}
		return 0;
	}
	string prefix = photo_url.substr(0, photo_url.rfind('/'));
	string suffix = photo_url.substr(photo_url.rfind('.'));
	char idname[1024];
	for (int i=1; lns_do_wget=='t' && i<=photo; ++i){
		sprintf(idname, "/%d%s", i, suffix.c_str());
		string name = idname;
		download_http_file((prefix+name).c_str(), (html_dir+name).c_str());
		if (file_exists((html_dir+name).c_str())) ++count;
		else cout<<"===== MISSING "<<prefix+name<<endl;
	}

	// record this project url
	string title = project_dir;
	if (title.find('/')!=string::npos)
		title = title.substr(title.rfind('/')+1);
	lns_record_project(("# "+title+"\n"+html_url+"\n\n").c_str());

	cout<<"==== total download photo "<<count<<endl;
	// rename project/[page] dir
	sprintf(cmd, "mv %s %s", html_dir, project_dir.c_str());
	printf("==== [CMD] %s\n", cmd);
	systemcmd(cmd, 'l');
	return count;
}


//-------------- do lns main -----------------

int lns_main(int argc, char *argv[])
{
	show_version();
	if (argc<2){
		printf("-h for help\n");
		return 0;
	}
	int id = parser_param(argc, argv);
	if (id!=0) return 0;
	remove_str(lns_list_dir, LNS_PATH_ILLEGAL);

	lns_record_project("# =========== begin download... ==========\n\n", 0);
	lns_load_exclude(lns_list_exclude);
	vector<string> urls = lns_load_urls(lns_list_file);
	lns_download_list(urls, lns_list_dir);
	return 0;
}

int main(int argc, char *argv[])
{
	lns_main(argc, argv);
	return 0;
}
